{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import joblib\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# trian data\n",
    "# data = joblib.load('../../2_feature_select_new/param_tune/train_data.lz4')\n",
    "# data_cf_sel = joblib.load('../cross_feature/data_cf_sel.lz4')\n",
    "# x_train = pd.concat([data,data_cf_sel],axis=1)\n",
    "# joblib.dump(x_train,'x_train.lz4',compress='lz4')\n",
    "\n",
    "\n",
    "# valid data\n",
    "valid_date = pd.read_csv('../../preprocess_data/valid_date.csv').drop(columns=['id','loan_hour'])\n",
    "valid_raw = joblib.load('../../preprocess_data_new/valid_nodup.lz4').drop(columns=['id','loan_dt'])\n",
    "valid_null = pd.read_csv('../../preprocess_data_new/valid_row_null.csv').drop(columns=['id'])\n",
    "valid_tag = pd.read_csv('../../2_feature_select_new/predict_tag_new/valid_tag.csv',usecols=['tag'])\n",
    "week_test = joblib.load('../../preprocess_data_discrete/week_test.lz4')\n",
    "valid_cf_sel = joblib.load('../cross_feature/valid_data/valid_cf_sel.lz4')\n",
    "\n",
    "x_valid = pd.concat([valid_date,valid_raw,valid_null,valid_tag,week_test,valid_cf_sel],axis=1)\n",
    "# joblib.dump(x_valid,'x_valid.lz4',compress='lz4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20000, 148)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_cf_sel.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((33465, 6853), (20000, 6853))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape,x_valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_month</th>\n",
       "      <th>loan_day</th>\n",
       "      <th>f1</th>\n",
       "      <th>f2</th>\n",
       "      <th>f3</th>\n",
       "      <th>f4</th>\n",
       "      <th>f5</th>\n",
       "      <th>f6</th>\n",
       "      <th>f7</th>\n",
       "      <th>f8</th>\n",
       "      <th>...</th>\n",
       "      <th>f5567*f6072</th>\n",
       "      <th>f6072*f1156</th>\n",
       "      <th>f3116*f2539</th>\n",
       "      <th>f3857*f1588</th>\n",
       "      <th>f3804*f3698</th>\n",
       "      <th>f3755*f4536</th>\n",
       "      <th>f660*f5127</th>\n",
       "      <th>f1927*f1150</th>\n",
       "      <th>f2145*f3819</th>\n",
       "      <th>f3688*f1927</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>41.833333</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>405.743351</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>82475.25</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.073164</td>\n",
       "      <td>1.209682</td>\n",
       "      <td>3.6765</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1164.000000</td>\n",
       "      <td>0.020571</td>\n",
       "      <td>241.724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>31</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>138.166164</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>167910.60</td>\n",
       "      <td>6815.389908</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.015020</td>\n",
       "      <td>2.9580</td>\n",
       "      <td>0.547619</td>\n",
       "      <td>592.666667</td>\n",
       "      <td>0.031000</td>\n",
       "      <td>99.568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>370.230109</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>52227.20</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.390431</td>\n",
       "      <td>3.061026</td>\n",
       "      <td>10.6680</td>\n",
       "      <td>0.212245</td>\n",
       "      <td>1735.125000</td>\n",
       "      <td>0.019500</td>\n",
       "      <td>430.311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>21</td>\n",
       "      <td>132.500000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.190476</td>\n",
       "      <td>607.404153</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>89043.50</td>\n",
       "      <td>10335.488696</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.443338</td>\n",
       "      <td>0.076313</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.236593</td>\n",
       "      <td>805.937500</td>\n",
       "      <td>0.011250</td>\n",
       "      <td>51.580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>125.333333</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>85.816989</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>87957.20</td>\n",
       "      <td>28099.406729</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.749893</td>\n",
       "      <td>24.5100</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1735.200000</td>\n",
       "      <td>0.074400</td>\n",
       "      <td>93.267</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 6853 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   loan_month  loan_day          f1  f2   f3        f4          f5   f6   f7  \\\n",
       "0           3        20   41.833333 NaN  0.0  0.000000  405.743351  0.0  0.0   \n",
       "1           1        31    0.000000 NaN  0.0       NaN  138.166164  0.0  0.0   \n",
       "2           4        14    0.000000 NaN  0.0  0.000000  370.230109  0.0  0.0   \n",
       "3           3        21  132.500000 NaN  0.0  0.190476  607.404153  0.0  0.0   \n",
       "4           3        10  125.333333 NaN  0.0       NaN   85.816989  0.0  0.0   \n",
       "\n",
       "    f8     ...       f5567*f6072   f6072*f1156  f3116*f2539  f3857*f1588  \\\n",
       "0  0.0     ...          82475.25      0.000000          NaN     0.073164   \n",
       "1  NaN     ...         167910.60   6815.389908          NaN     0.000000   \n",
       "2  NaN     ...          52227.20      0.000000          0.0     0.390431   \n",
       "3  NaN     ...          89043.50  10335.488696          0.0     0.443338   \n",
       "4  NaN     ...          87957.20  28099.406729          NaN     0.000000   \n",
       "\n",
       "   f3804*f3698  f3755*f4536  f660*f5127  f1927*f1150  f2145*f3819  f3688*f1927  \n",
       "0     1.209682       3.6765    0.000000  1164.000000     0.020571      241.724  \n",
       "1     2.015020       2.9580    0.547619   592.666667     0.031000       99.568  \n",
       "2     3.061026      10.6680    0.212245  1735.125000     0.019500      430.311  \n",
       "3     0.076313       0.0000    0.236593   805.937500     0.011250       51.580  \n",
       "4     1.749893      24.5100    0.000000  1735.200000     0.074400       93.267  \n",
       "\n",
       "[5 rows x 6853 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_month</th>\n",
       "      <th>loan_day</th>\n",
       "      <th>f1</th>\n",
       "      <th>f2</th>\n",
       "      <th>f3</th>\n",
       "      <th>f4</th>\n",
       "      <th>f5</th>\n",
       "      <th>f6</th>\n",
       "      <th>f7</th>\n",
       "      <th>f8</th>\n",
       "      <th>...</th>\n",
       "      <th>f5567*f6072</th>\n",
       "      <th>f6072*f1156</th>\n",
       "      <th>f3116*f2539</th>\n",
       "      <th>f3857*f1588</th>\n",
       "      <th>f3804*f3698</th>\n",
       "      <th>f3755*f4536</th>\n",
       "      <th>f660*f5127</th>\n",
       "      <th>f1927*f1150</th>\n",
       "      <th>f2145*f3819</th>\n",
       "      <th>f3688*f1927</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>57.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>442.180079</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>18200.5</td>\n",
       "      <td>177.609659</td>\n",
       "      <td>1851.98556</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.160341</td>\n",
       "      <td>1.484</td>\n",
       "      <td>0.0</td>\n",
       "      <td>979.692308</td>\n",
       "      <td>0.109667</td>\n",
       "      <td>206.960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>130435.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>152.425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>27</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21.197484</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.154</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3879.000000</td>\n",
       "      <td>0.086000</td>\n",
       "      <td>411.174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>63639.0</td>\n",
       "      <td>31052.894405</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.447573</td>\n",
       "      <td>1.820224</td>\n",
       "      <td>8.400</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1268.000000</td>\n",
       "      <td>0.058667</td>\n",
       "      <td>174.984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>23</td>\n",
       "      <td>0.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>43.675890</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>13043.5</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.402451</td>\n",
       "      <td>1.650</td>\n",
       "      <td>0.0</td>\n",
       "      <td>403.111111</td>\n",
       "      <td>0.011111</td>\n",
       "      <td>769.136</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 6853 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   loan_month  loan_day    f1    f2   f3   f4          f5   f6   f7   f8  \\\n",
       "0           3         7  57.0   0.0  0.0  0.0  442.180079  0.0  0.0  NaN   \n",
       "1           3         7   0.0   NaN  0.0  0.0    0.000000  0.0  0.0  NaN   \n",
       "2           1        27   0.0   NaN  0.0  NaN   21.197484  0.0  0.0  NaN   \n",
       "3           3        29   0.0   NaN  0.0  NaN    0.000000  0.0  0.0  0.0   \n",
       "4           1        23   0.0  41.0  0.0  0.0   43.675890  0.0  0.0  NaN   \n",
       "\n",
       "      ...       f5567*f6072   f6072*f1156  f3116*f2539  f3857*f1588  \\\n",
       "0     ...           18200.5    177.609659   1851.98556     0.000000   \n",
       "1     ...          130435.0      0.000000          NaN     0.000000   \n",
       "2     ...               NaN      0.000000          NaN     0.000000   \n",
       "3     ...           63639.0  31052.894405          NaN     0.447573   \n",
       "4     ...           13043.5      0.000000          NaN     0.000000   \n",
       "\n",
       "   f3804*f3698  f3755*f4536  f660*f5127  f1927*f1150  f2145*f3819  f3688*f1927  \n",
       "0     3.160341        1.484         0.0   979.692308     0.109667      206.960  \n",
       "1     0.000000        0.000         0.0   650.000000     0.000000      152.425  \n",
       "2     0.000000        0.154         0.0  3879.000000     0.086000      411.174  \n",
       "3     1.820224        8.400         0.0  1268.000000     0.058667      174.984  \n",
       "4     9.402451        1.650         0.0   403.111111     0.011111      769.136  \n",
       "\n",
       "[5 rows x 6853 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_valid.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
